#!/bin/bash
#
#
#    Run hybrid WARP3D on Linux (MPI + OpenMP)
#    =========================================
#
#    Last updated: June 16, 2013 (RHD)
#
#    This bash script runs the "hybrid" version of WARP3D that
#    includes MPI, OpenMP and the hypre solver. The Intel MKL libraries
#    (BLAS) are used by hypre.
#
#    *** The finite element model must have domain decomposition ***
#    *** defined in the blocking input                           ***
#
#    Solver available:   hypre
#
#
#    How to set up:
#    -------------
#
#      Processors: number of physical chips located across one or more
#                  computers - each processor contains one or more cores
#
#      Cores: a CPU residing on a processor. A core runs a single
#             thread (we recommend not over assigning threads to cores
#             that triggers hyperthreading).
#
#      Method 1 (recommended)
#        number of model doamins = number of processors = number of MPI ranks
#        OMP_NUM_THREADS = number of cores / processor to be used
#        MKL_NUM_THREADS = OMP_NUM_THREADS
#
#      Method 2
#        number of model domains = number of all cores to be used = number of MPI ranks
#        OMP_NUM_THREADS = MKL_NUM_THREADS = 1 (MPI ranks saturate each core)
#
#
#      User specifies # MPI ranks and OMP_NUM_THREADS -- script here
#      sets MKL_NUM_THREADS = OMP_NUM_THREADS
#
echo " "
echo ">> Running MPI + OpenMP (hybrid) version of WARP3D on Linux (64)..."
echo " "
#
if [ $# != "2" ]; then
echo ">> Usage:"; echo " "
echo "  warp3d_hybrid_linux  <parms> ( < input ) ( > output )"; echo " "
echo "      Requirements:"
echo " "
echo "        - model must have domain decomposition w/ domains assigned in blocking"
echo "        - the hypre solver must be used"
echo " "
echo "      There are 2 parameters:"
echo " "
echo "      (1) number of MPI processes (ranks) = # domains defined in blocking"
echo " "
echo "      (2) number of OpenMP threads per rank"
echo " "
echo "      Guidelines:"
echo "      -----------"
echo "  "
echo "       1. number of MPI ranks (NP) must = number of domains defined in model input"
echo "          We recommend using NP = number of physical processors available"
echo " "
echo "       2. number of OpenMP threads = number of cores to use on each processor"
echo " "
echo "       - or - "
echo " "
echo "       1. number of MPI ranks (NP) must = number of domains defined in model input"
echo "          and np = number of cores available to use"
echo " "
echo "       2. number of OpenMP threads = 1"
exit 1
fi
#
if [ -z "$WARP3D_HOME" ]; then
   printf "[ERROR]\n"
   printf "... An environment variable WARP3D_HOME is not set.\n"
   printf "Quitting...\n"
   exit 1
fi
#
#           set executable file for MPI + OpenMP + hypre solver
#           version of warp3d
#
warp3d_exe="$WARP3D_HOME/run_linux_em64t/warp3d.mpi_omp"
#
#           set LD_LIBRARY_PATH
#
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$WARP3D_HOME/linux_packages/lib
#
#
export NUM_WARP3D_RANKS=$1
export OMP_NUM_THREADS=$2
export MKL_NUM_THREADS=$2
#
if [ $1 = "0" -o $2 = "0"  ]; then
 echo " "
 echo "[ERROR]"
 echo " "
 echo "... values for 2 parameters must be > 0 ..."
 echo " "
 echo "Quitting..."; echo " "
 exit 1
fi
#
#           increase the allowable size of the runtime stack. needed
#           for MPI jobs that also use threads
#
ulimit -s 100000
#
#
#   =======  no need to change below here =======================
#
#
export MPI_TYPE_MAX=4096
export I_MPI_PIN_DOMAIN=omp
export I_MPI_PIN_HOST=1
export I_MPI_FABRICS=shm:tcp
#
#       touch core file and make it non-writable in case we dump
#
touch core &> /dev/null
chmod ugo-rwx core
#
#       start WARP3D on each of the MPI processes using message
#       passing based on shared memory
#
echo " Starting WARP3D with MPI + OpenMP hybrid execution..."
echo "    o Number MPI processes:             " $NUM_WARP3D_RANKS
echo "    o Number OpenMP threads each rank:  " $OMP_NUM_THREADS
#
# Let the OS sort out getting MPI daemons started
#
#
mpirun -np $NUM_WARP3D_RANKS $WARP3D_HOME/run_linux_em64t/warp3d.mpi_omp
#
# Cleanup the core file (if it was created)
#
/bin/rm -f core
exit








